import os, sys, time, asyncio, aiohttp

import re

import pymysql
import tqdm

# 保存目录
DEST_DIR = 'downloads/'
headers = {
    "User-Agent": 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.13 Safari/537.36',
    "Referer": "http://www.v2ex.com/signin"
}


# 获取链接,下载文件
async def fetch(session: aiohttp.ClientSession, url: str, path: str, flag: str):
    async with session.get(url) as resp:
        with open(path, 'wb') as fd:
            while 1:
                chunk = await resp.content.read(1024)  # 每次获取1024字节
                if not chunk:
                    break
                fd.write(chunk)
    return flag


async def begin_download(sem, session: aiohttp.ClientSession, url: str, path: str, flag: str):
    # 控制协程并发数量
    with (await sem):
        return await fetch(session, url, path, flag)


async def download(sem: asyncio.Semaphore):
    tasks = []
    async with aiohttp.ClientSession() as session:
        for flag in textList:
            flag = flag.replace('\n', '')
            # 创建路径以及url
            pathUrl = re.findall('com/(.*?)([^/]*)$', flag)
            os.makedirs('%s%s' % (DEST_DIR, pathUrl[0][0]), exist_ok=True)
            path = os.path.join('%s%s%s' % (DEST_DIR, pathUrl[0][0], pathUrl[0][1]))
            tasks.append(asyncio.ensure_future(begin_download(sem, session, flag, path, flag)))
        # 等待返回结果
        tasks_iter = asyncio.as_completed(tasks)
        # 创建一个进度条
        fk_task_iter = tqdm.tqdm(tasks_iter, total=len(textList))
        for coroutine in fk_task_iter:
            res = await coroutine
            print(res, '下载完成')


if __name__ == '__main__':
    with open('E:/en/xx.txt', mode='r', encoding='utf-8') as f:
        textList = f.readlines()
    # 创建目录
    os.makedirs(DEST_DIR, exist_ok=True)
    # 获取事件循环
    lp = asyncio.get_event_loop()
    start = time.time()
    # 创建一个信号量以防止DDos
    sem = asyncio.Semaphore(4)
    lp.run_until_complete(download(sem))
    end = time.time()
    lp.close()
    print('共耗时:', end - start)
    print("完")
